{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import gym\n",
    "import gym_custom_envs\n",
    "import random\n",
    "import time\n",
    "from matplotlib.pylab import plt\n",
    "import seaborn as sns\n",
    "env = gym.make(\"DoubleIntegrator-v0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[20, 20] [0.2 0.2] (20, 20)\n"
     ]
    }
   ],
   "source": [
    "N = [env.dense_parameter]*len(env.observation_space_high)\n",
    "delta_T = (env.observation_space_high - env.observation_space_low)/N\n",
    "\n",
    "def getDiscrete(s):\n",
    "    dis_s = (s - env.observation_space_low)/delta_T\n",
    "    return tuple(dis_s.astype(np.int))\n",
    "print(N, delta_T, getDiscrete(np.array([2,2])))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(21, 21, 2)\n",
      "(21, 21)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAAD8CAYAAADUv3dIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAdK0lEQVR4nO3de7gcVZnv8e+PhHC/COGaBMJgUDnqIOQgjhfQgAbwIc6gM4AeLjLGcUQBZ1QY5ijgoyY6iJ4ZRgk3FZQ7w0SNXESBOTMCCRgw4RoQSIIQbsJ4UCF7v+ePqq3NdndX9e7u6tWV34ennvRlrapVVPfaq1fVW68iAjMzq8Z6/W6Amdm6xJ2umVmF3OmamVXIna6ZWYXc6ZqZVcidrplZhdzpmpk1Iel8SWskLWvyviT9H0krJN0lac+idbrTNTNr7pvA7BbvHwjMyJe5wNeLVuhO18ysiYi4GXimRZE5wLcjcwuwpaQdWq1zYjcb2IRD3sysLHW6gpeeeqh0nzNpm10/TDZCHbEgIha0sbkpwMqG56vy137ZrEIVnS4TJ00pXXbti6vHVaed8uOpM952Abz01EOl66w/+U/aKj9Sp6p9qeK4VLX/7R6X8dRJeV9SO5Yj7apS3sG208l2rJJO18ysMsNDVW5tNTCt4fnU/LWmPKdrZvUytLb80rmFwJH5VQz7AM9FRNOpBSgx0pX0arLJ4pHfFquBhRFxT6etNTPrtojhrq1L0sXAfsBkSauAzwLrZ9uJbwCLgIOAFcALwDFF62zZ6Ur6NHA4cAlwW/7yVOBiSZdExLxx7YmZWa8Md6/TjYjDC94P4KPtrLNopHss8D8i4qXGFyV9BVgOjNnpSppLfkbw7LPPbqc9Zmad6eJItxeK5nSHgR3HeH2H/L0xRcSCiJgZETPnzp3brJiZWfcND5Vf+qBopHsCcIOkB/jDtWg7Aa8Ejutlw8zMxiXxkW7LTjcirpG0G7A3Lz+Rtjgi+vNnwsyshejOVQk9owpypDkizczK6jgi7XcP/FfpPmeDGX/W8fbaVUlwRBWRP1VFpKW6L1VEJI2nzngisnpdZ7zHBdLb/yr3parvWMcGeXrBzGzg9OkEWVnudM2sXjzSNTOrUOIn0tzpmlm9dDEirRfc6ZpZraR+Nas7XTOrF8/pmplVyNMLZmYVSnyk64g0M0tJxxFiv73t8tJ9zoZ7v6+eEWlmZpXx9EK64ZZ1SkyZauhor4/LyHbqdCxTTUxZ1eeyY4lPL3ika2b1kvhItzAxpaRXS5oladNRr8/uXbPMzMZpeLj80gctO11JHwf+HfgYsEzSnIa3v9DLhpmZjUcMvVR66Yeike6HgL0i4j1kGTH/t6Tj8/eanvWTNFfSEklLFixY0J2WmpmVEcPllz4omtNdLyJ+DRARD0vaD7hC0s606HQjYgEw0ttGuxPwZmbjNuBzuk9I2mPkSd4BvxuYDLyulw0zMxuXAR/pHgm87D5pEbEWOFKSc6ubWXoSH+kWJaZc1eK9/+x+c8zMOpT4dboOAzazlHQclvubH3y1dJ+z0cEn1DMMONXIn6qSOdYpyWYVyQzX9WNZp31xYso/5og0M6uXQZ7TNTMbOB7pmplVyCNdM7MKeaRrZlahtU7BbmZWnd5fBtsRd7pmVi+e0zUzq1Dina4j0swsJZ1HpF10SvmItA98vuX28mQNXwMmAOdGxLxR7+8EfAvYMi9zUkQsarVOj3TNrF6GhrqyGkkTgLOAA4BVwGJJCyPi7oZi/whcFhFfl7Q7sAiY3mq9tUlMWVW4ZRV1Uk6ymdpxgWrDgFPbf4cBj6F70wt7Aysi4iEASZcAc4DGTjeAzfPHWwCPFa3UI10zq5c2Ol1Jc4G5DS8tyJMwAEwBVja8twp446hVnApcJ+ljwCbA/kXbLExMOUYjv91uHTOzyrRxE/OIWBARMxuWdvOLHQ58MyKmAgcBF0pq2a+2HOlKWjj6JeDtkrYEiIhD2mygmVlPxXDXzt2vBqY1PJ+av9boWGA2QET8VNKGZJl11jRbadFIdyrwPPAV4Ix8+e+Gx2NyYkoz65vupWBfDMyQtIukScBhwOiB6KPALABJrwE2BJ5stdKiOd2ZwPHAKcAnI2KppN9ExE2tKjkxpZn1TZeuXoiItZKOA64luxzs/IhYLul0YElELAT+DjhH0olkJ9WOjoLrcIvS9QwDZ0q6PP/3iaI6ZmZ91cXgiPya20WjXvtMw+O7gTe3s85SHWieK+19kg4mm24wM0uTI9IckWZmpXUckfbCVz9cus/Z+ISz65kjzcysMomPdB2R1ma7Ut2XVCPSeh1dNp46jkhLO/lrx7p3yVhPeKRrZvXSpasXesWdrpnVSnh6wcysQp5eMDOrkBNTmplVyCNdM7MKrfWJNDOz6nh6wcysQolPLzgM2MxS0nFY7q9PPrR0n7PpF6+sZxhwnfI3pbov62p03UidOkVxVdGu8dSpal86lvhI19MLZlYv7nTNzCqUeBhwy3Q9kt4oafP88UaSTpP0PUnzJW1RTRPNzMqL4Si99ENRjrTzgRfyx18jy+s+P3/tgh62y8xsfIaj/NIHRdML60XE2vzxzIjYM3/8fyUtbVapMZf82Wef3XkrzczKSvyGN0Uj3WWSjskf3ylpJoCk3YCXmlVqzCU/d+7cLjXVzKyExEe6RZ3uXwP7SnoQ2B34qaSHgHPy98zM0pJ4p1uUDfg54Oj8ZNoueflVEfFEFY0zM2tXDKU9veCINDNLSccRYs8fe0DpPmfz866vZ0SamVlV+nUpWFnJhgGnlgCxytBRh8Gu22HQ6/q+dMydrplZhdKe0nWna2b1EmvT7nXd6ZpZvaTd57rTNbN68Yk0M7MqeaRrZlYdj3TNzKqU+EjXEWlmlpKOI8SePnjf0n3O1j+4yRFpZmadSDwDe30i0qpKTJlq5E9VSTZT25fx1KnT/jsx5Ri62OlKmk2WwGECcG5EzBujzF8Cp5L9qr8zIo5otU6PdM2sVro10pU0ATgLOABYBSyWtDAi7m4oMwM4GXhzRDwradui9RblSJsk6UhJ++fPj5D0L5I+Kmn9TnbIzKwXYrj8UmBvYEVEPBQRLwKXAHNGlfkQcFZEPAsQEWuKVlo00r0gL7OxpKOATYGrgFl5g44qbLaZWYViqPy5scbUYrkFEbEgfzwFWNnw3irgjaNWsVu+nv8km4I4NSKuabXNok73dRHxekkTgdXAjhExJOki4M4yO+IcaWZWpXamF/IOdkFhweYmAjOA/YCpwM2SXhcRv2pWoShdz3qSJgGbARuTZQMG2ABoOr3gHGlm1i8xrNJLgdXAtIbnU/PXGq0CFkbESxHxC+B+sk64qaJO9zzgXmApcApwuaRzgMVk8xtmZknp4pzuYmCGpF3ywedhwMJRZa4mG+UiaTLZdEPLyzyKcqSdKenS/PFjkr4N7A+cExG3FTbZzKxiEd2Jd4iItZKOA64lm689PyKWSzodWBIRC/P33inpbmAI+GREPN1qvYWXjEXEYw2PfwVc0cF+mJn1VDeDIyJiEbBo1GufaXgcwCfypRSHAZtZSjoepj46c1bpPmenJTfUMww41WiZqvKKVZHvLdUopl7n1RrZzrqeVyzVfHf9yJFW4gRZXzkizcxqxZ2umVmFej9j2hl3umZWKx7pmplVqFuXjPWKO10zq5WhNu690A/udM2sVjzSNTOrkOd0zcwqlPrVC45IM7OUdDxMvXvXg0v3Obs/+IN6RqSZmVVlaLjo5on9VZvElCmHjqaaADDVMGAnpkzzWA5KYsrUpxc80jWzWhn21QtmZtXxJWNmZhVaJ6cXnJjSzPol9emFlqf5JG0haZ6keyU9I+lpSffkr23ZrJ4TU5pZvwwNr1d66YeirV4GPAvsFxFbRcTWwNvz1y7rdePMzNoVbSz9UNTpTo+I+RHx+MgLEfF4RMwHdu5t08zM2jccKr30Q1Gn+4ikT0nabuQFSdtJ+jSwsrdNMzNrX4RKL/3QMgxY0iuAk4A5wLb5y0+Q5X6fFxHPlthG4ucSzSwhHfeE/7H9e0v3OW99/Iq0woDzTvXT+fIyko4BLii1kRolzUs1iinVfen1cRmpk2pEWp0iJauKrutUdN5v91Qnp+9O61orzMy6ZG2o9NIPLUe6ku5q9hawXZP3zMz6JvWRblFwxHbAu8guEWsk4L960iIzsw4M97sBBYo63e8Dm0bE0tFvSLqxJy0yM+vAQI90I+LYFu8d0f3mmJl1ZtBHumZmA2VokEe6ZmaDJvG8lO50zaxehhMf6ToxpZmlpOMe8+rtjyjd57zn8e+mFZHWtY2s41FcqUb+VLH/vT4uI21LNYor5X1JNbquUz6RZmZWoWGlPb3gTtfMamWo3w0okHaCeDOzNg2r/FJE0mxJ90laIemkFuUOlRSSZhat0yNdM6uVbl29IGkCcBZwALAKWCxpYUTcParcZsDxwK1l1luUI21zSV+UdKGkI0a9968t6s2VtETSkgULFpRph5lZV3QxXc/ewIqIeCgiXgQuIbu3+GifA+YDvy3TvqLphQvILuG4EjhM0pWSNsjf26dZJSemNLN+aWd6oXGAmC+NHdYUXp4hZ1X+2u9J2hOYFhE/KNu+oumFXSPi0Pzx1ZJOAX4s6ZCyGzAzq1I7l4xFxAJgXD/HJa0HfAU4up16RZ3uBpLWi4jhvIGfl7QauBnYdDwNNTPrpaHuXTG2GpjW8Hxq/tqIzYDXAjcqu0xte2ChpEMiYkmzlRblSPsScF1E/GjU67OBf46IGSUa7og0Myur4y7znKkfKN3nfGjVRU23J2kicD8wi6yzXQwcERHLm5S/Efj7Vh0uFN/a8VNNXr9G0hda1TUz64duRaRFxFpJxwHXAhOA8yNiuaTTgSURsXA86x33vRckPRoRO5UoGnUKA0413LKqMODUQprHU6fKMOiUk2wmGtLc8Uj3G9PKj3T/ZmXzkW6vOEeamdXKoN97wTnSzGygpB4G7BxpZlYrA30Tc+dIM7NBM+jTC2ZmA8WdrplZhVIPDHCna2a1MtBzumZmgyb1qxecmNLMUtLxOPXzO7+/dJ9zyiPfSSs4omsbqU+0TLLJDKuKYkotUhDql8wx1WPpxJTd4ekFM6uV1H9au9M1s1rxSNfMrEJrlfZYt+1swJK2LVHGOdLMrC+6mCOtJ4ruMrbV6JeA2yS9gezKh2fGqjcqBUb87XGnddxQM7MyBn164SngkVGvTQHuIPtD0Z3TjWZmXTKc+Km0oumFTwL3AYdExC4RsQuwKn/sDtfMkjPQ0wsRcYakS4EzJa0EPkv6V2SY2Tos9emF0hFpedr1fwCmR8T2bWzDnbSZldVxhNiJ0w8r3eec+fAl6UakRcRCSdcDuwJIOiYiLihTt05RXHWKYqpiX1KOSKtTFFcVx7+qPISdSn2k29YlYxHxm4hYlj/1JQlmlpxo479+cGJKM6uV1Ee6TkxpZrWS+iVjTkxpZrWSdpfrxJRmVjNrE+92fcMbM6uVfp0gK8udrpnVyqCfSDMzGyipj3SdI83MUtJxhNhR0w8t3ed86+Er041IMzMbBEO9H0h2JNnElKmFm1aZmDLlfaniuFS1/3UKA65TqH2nBv06XTOzgZL6nK47XTOrFV+9YGZWodSnF8aTmHLrEmWcmNLM+qKbdxmTNFvSfZJWSDppjPc/IeluSXdJukHSzkXrbNnpSponaXL+eKakh4BbJT0iad+mOx2xICJmRsTMuXPnFu6YmVm3DEWUXlqRNAE4CzgQ2B04XNLuo4r9DJgZEa8HrgC+VNS+opHuwRHxVP74y8BfRcQrgQOAM4pWbmZWtWGi9FJgb2BFRDwUES8ClwBzGgtExE8i4oX86S3A1KKVFnW6EyWNzPtuFBGL8w3dD2xQtHIzs6oNt7E0ToXmS+NP8ynAyobnq/LXmjkW+GFR+4pOpP0rsEjSPOAaSV8DrgLeAfzR7R7NzPqtnUvGImIB0PGJJ0kfAGYCTaddf1+2KAxY0n7AR4DdyDrplcDVwPkRsbZEe9I+lWhmKek4LPegnQ4q3ecsenRR0+1JehNwakS8K39+MkBEfHFUuf2Bfwb2jYg1RdssvGQsIm4EbhyjQccAySSmrFMU07oekVanZI5VRXGlmpiyHxFpXbyfzGJghqRdgNXAYcDL7iMu6Q3A2cDsMh0ujOOSsQZOTGlmyRkiSi+t5L/kjwOuBe4BLouI5ZJOl3RIXuzLwKbA5ZKWSlpY1D4npjSzWulmcERELAIWjXrtMw2P9293nU5MaWa1UsHtajvixJRmViuphwE7MaWZ1YrvMmZmViHfxNzMrEIDPb1gZjZoUu90nZjSzFLScUTaPjvuV7rPueWxG+uZmDLViLSqIn+qyKuVahRTr6PLxlOnyoi0lCMlUzuWI/vSqdRHup5eMLNa8dULZmYVGoq0s6S50zWzWhn0iDQzs4GS+pxuUY60mZJ+IukiSdMkXS/pOUmL81uaNavnxJRm1hfdTEzZC2UyR3wW2JLsBjcnRsQBkmbl771prEqj7sYe7Z71NDMbr+HEpxeK7qe7fkT8MCIuBiIiriB7cAOwYc9bZ2bWpkEf6f5W0juBLYCQ9J6IuDpPvz7U++aZmbUn9asXWkakSfpTsjzuw8CJZLnSjiJLXfGhiChzT920x/pmlpKOI8R222Zm6T7n/ieXVB6RNu4wYEnHRESZHGnudM2srI47wRnb7FW6z3ngydsHKgz4NHqYmLKKBIhVhE6Op05VYcBVHJde78t4tpN6Ys5Uw9OrOi6dSv1EmnOkmVmtDHoYsHOkmdlAGYq0z/E7R5qZ1cpAhwE7R5qZDZrUw4B97wUzq5WBHumamQ2agb56wcxs0Az61QtmZgNloMOAuyTtPztmlpKOI8Qmb75b6T7nqefvH6iItNJSjeKqUwLAVKOYqkpMWafowqqi61Ldl055TtfMrEK+esHMrEK+TtfMrEKpj3SLcqRtIWmepHslPSPpaUn35K9tWVUjzczKGorh0ks/FKXruYzsZjf7RcRWEbE18Pb8tcuaVXJiSjPrl+GI0ks/FE0vTI+I+Y0vRMTjwHxJH2xWyYkpzaxfBnp6AXhE0qck/f7euZK2k/RpYGVvm2Zm1r5uJqaUNFvSfZJWSDppjPc3kHRp/v6tkqYXrbOo0/0rYGvgJknPSnoGuBHYCvjLwhabmVUsIkovrUiaAJwFHAjsDhwuafdRxY4Fno2IVwJnAvMpUBiRJunVwFTgloj4dcPrsyPimqIN4Ig0Myuv4wixiZOmlO5z1r64uun2JL0JODUi3pU/PxkgIr7YUObavMxPJU0EHge2iRYda9HVCx8H/h04DlgmaU7D218o3qVsNc0WSR9u9X6n5VOuk2q7vC9ptmsd2peOrX1xtcoujSf982Vuw6qm8PJp1FX5a4xVJiLWAs+RzQ40VzD0/jlZ5giA6cAS4Pj8+c/aGcY3Wf+SXpZPuU6q7fK+pNmudX1f+rEA7wXObXj+v4B/GVVmGTC14fmDwORW6y26emG9yKcUIuJhSfsBV0jamS79VTIzS9RqYFrD86n5a2OVWZVPL2wBPN1qpUUn0p6QtMfIk7wDfjcwGXhduXabmQ2kxcAMSbtImgQcBiwcVWYhcFT++L3AjyMf8jZTNNI9Eljb+EJk8xZHSjq7bMtbaDdyYjyRFqnWSbVd46mTarvGUyfVdo2nTqrtGm+dSkXEWknHAdcCE4DzI2K5pNPJpkcWAucBF0paATxD1jG3VMX9dM3MLFc0vWBmZl3kTtfMrEJ96XSLQuvGKH++pDWSlrWxjWmSfiLpbknLJR1fUH5DSbdJujMvf1ob25og6WeSvl+y/MOSfi5pqaQlJetsKemK/I5v9+QXbjcr+6p83SPL85JOKLGNE/N9XybpYkkblqhzfF5+ebNtjHX8JG0l6XpJD+T/vqJEnffl2xmWNLNE+S/n/7/ukvRvGnVnvCZ1PpeXXyrpOkk7FtVpeO/vJIWkySW2c6qk1Q3H6KCibUj6WL4/yyV9qcQ2Lm1Y/8OSlpaos4ekW0Y+m5L2LlHnTyX9NP9Mf0/S5g3vjfk9LDr+tdaHa98mkF3L9ifAJOBOYPeCOm8D9gSWtbGdHYA988ebAfe32g7ZJXAj1ySvD9wK7FNyW58Avgt8v2T5hym4lm+MOt8C/jp/PAnYso3/348DOxeUmwL8Atgof34ZcHRBndeSXae4MdlJ2R8Bryxz/IAvASflj08C5peo8xrgVWSh6DNLlH8nMDF/PL/kNjZvePxx4BtlPotklw1dCzwy+tg22c6pwN+X/byT3d3vR8AG+fNt2/mOAGcAnymxneuAA/PHBwE3lqizGNg3f/xB4HMN7435PSw6/nVe+jHS3RtYEREPRcSLwCXAnFYVIuJmsjODpUXELyPijvzxfwP38MfRJI3lI/4Q5rx+vhSeZZQ0FTgYOLed9rVD0hZkH/bzACLixYj4Vcnqs4AHI+KREmUnAhspu95wY+CxgvKvAW6NiBciu6rlJuAvRhdqcvzmkP0hIf/3PUV1IuKeiLhvrIY0KX9d3i6AW8iusyyq83zD000Y9Rlo8Vk8E/jU6PIFdcbUpPxHgHkR8bu8zJqy25AksnulXFyiTgAjI9UtGPUZaFJnN+Dm/PH1wKEN5Zt9D1se/zrrR6dbJrSuqyRNB95ANnptVW5C/hNsDXB9RLQsn/sq2ZetnTsiB3CdpNv18rDDZnYBngQuyKcxzpW0ScltHcaoL9uYDYpYDfwT8CjwS+C5iLiuoNoy4K2Stpa0MdnIaFpBnRHbRcQv88ePA9u1KtwFHwR+WKagpM9LWgm8H/hMifJzgNURcWebbToun8o4v8TP693I/l/fKukmSf+zje28FXgiIh4oUfYE4Mv5/v8TcHKJOsv5w8DpfTT5DIz6HlZ9/JNR+xNpkjYFrgROGDWK+SMRMRQRe5CNiPaW9NqCdb8bWBMRt7fZrLdExJ5kdy/6qKS3FZSfSPaT7usR8Qbg/5H9JGtJ2QXdhwCXlyj7CrIvzi7AjsAmkj7Qqk5E3EP2s/064BpgKTBUtK0x1hP08MZIkk4hu978OyXbc0pETMvLH1ew7o2Bf6BE5zzK14FdgT3I/sidUVB+Itnd/fYBPglclo9gyzicEn94cx8BTsz3/0TyX1cFPgj8raTbyaYQXhxdoNX3sNfHPzX96HTLhNZ1haT1yQ70dyLiqrL18p/uPwFmFxR9M3CIpIfJpkneIemiEutfnf+7Bvg3simXVlYBqxpG3leQdcJFDgTuiIgnSpTdH/hFRDwZES8BVwF/VlQpIs6LiL0i4m1kGUXuL7EtyKIddwDI/11TUH5cJB1NFkX5/vzL3Y7v0PBTuYldyf5Q3Zl/DqYCd0javlWliHgi/yM/DJxDuc/AVfk02G1kv6wmF9Qhnyr6C+DSorK5o8iOPWR/rIvaRUTcGxHvjIi9yDr3B0e1YazvYSXHP0X96HTLhNZ1LB8FnAfcExFfKVF+m5Gz25I2Ag4A7m1VJyJOjoipETGdbD9+HBEtR4eSNpG02chjspM9La/KiCxbx0pJr8pfmgXcXbRPtDfCeRTYR9LG+f+7WWTzby1J2jb/dyeyL/d3S26vMXzyKLK72XWVpNlkUz+HRMQLJevMaHg6h+LPwM8jYtuImJ5/DlaRnTh6vGA7OzQ8/XMKPgPA1WQn05C0G9nJ1KcK6kD2x/TeiFhVoixkc7j75o/fARROSTR8BtYD/hH4RsN7zb6HPT/+yerH2Tuyub/7yf4inlKi/MVkP8FeIvtQH1uizlvIfrLcRfazdylwUIvyrwd+lpdfxqgzvSW2tx8lrl4gu2rjznxZXmb/83p7kN3l7S6yL+ArCspvQnbjjS3a2IfTyDqZZcCF5GfKC+r8B9kfgDuBWWWPH9nt724g+1L/CNiqRJ0/zx//DngCuLag/Aqy8wcjx3/0lQhj1bky3/+7gO8BU9r5LDLGlSlNtnMh2V387iLrgHYoKD8JuChv2x3AO8q0C/gm8DdtHJe3ALfnx/NWYK8SdY4n+z7fD8wjj3Rt9T0sOv51XhwGbGZWodqfSDMzS4k7XTOzCrnTNTOrkDtdM7MKudM1M6uQO10zswq50zUzq9D/BxfExnmW3EztAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "J = np.random.uniform(low=0, high=1, size=[env.dense_parameter+1, env.dense_parameter+1, len(env.observation_space_high)])\n",
    "# J = np.zeros((env.dense_parameter+1, env.dense_parameter+1, len(env.observation_space_high)))\n",
    "print(J.shape)\n",
    "action_map = np.array([[random.choice([0, env.umax]) for i in range(J.shape[0])] for i in range(J.shape[0])])\n",
    "# action_map = np.ones((J.shape[0], J.shape[0]))\n",
    "print(action_map.shape)\n",
    "ax = sns.heatmap(action_map, linewidth=0.5)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output\n",
    "episodes = 50000\n",
    "lr = 0.1\n",
    "dis = 0.9\n",
    "for i in range(episodes):\n",
    "    s = getDiscrete(env.reset())    \n",
    "    done = False\n",
    "    print(i)\n",
    "#     print(s)\n",
    "    while not done:\n",
    "        a = action_map[s[0], s[1]]\n",
    "        ns, cost, done, _ = env.step(a)\n",
    "        ns = env.get_discrete(ns)\n",
    "        s = ns\n",
    "#         if i%500 == 0: \n",
    "#             env.render()\n",
    "        if not done:\n",
    "            temp_a = np.argmax(J[ns[0], ns[1]])\n",
    "#             print(temp_a)\n",
    "            J[ns[0], ns[1], temp_a] += (np.max(dis*J[ns[0], ns[1]]) + cost - J[ns[0], ns[1], temp_a])\n",
    "#             J[ns[0], ns[1], temp_a] /= np.max(J)\n",
    "#             if temp_a == 0:\n",
    "#                 temp_a = -1\n",
    "            action_map[ns[0], ns[1]] = temp_a     \n",
    "#             ns1, cost1, done,_ = env.step(env.umin)\n",
    "#             ns2, cost2, done,_ = env.step(env.umax)\n",
    "#             J[ns1[0], ns1[1]]\n",
    "#             ns1 = env.get_discrete(ns1)\n",
    "#             ns2 = env.get_discrete(ns2)\n",
    "#             if np.argmin([cost1, cost2]) == 0:\n",
    "#                 action_map[ns[0], ns[1]] = -1.\n",
    "#             else:\n",
    "#                 action_map[ns[0], ns[1]] = 1.\n",
    "            \n",
    "#             J[s[0], s[1]] = np.min([cost1, cost2]) + cost\n",
    "#             done = False\n",
    "#         print(action_map)\n",
    "#         print(J)\n",
    "        clear_output(wait=True)\n",
    "#         time.sleep(0.05)\n",
    "\n",
    "#     print(s)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAAD8CAYAAADUv3dIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAdQUlEQVR4nO3debwcZZ3v8c+XhLAvskMSCINB5eoMQi7iuIAGNIIv4oiOgF4WGY7jFQWcUWGYi4AvlaiI3hlGEzYVlJ1hokYWUWDujEACEszCEhDICYawCdeLGnLO7/5RdcamOd1VvVVXV75vXvVKd/XzVD1Fd//O00/Vrx5FBGZmVowN+t0AM7P1iYOumVmBHHTNzArkoGtmViAHXTOzAjnompkVyEHXzKwBSRdLWiNpSYPXJel/S1oh6T5J+2Rt00HXzKyx7wCzmrz+HmB6ugwB38raoIOumVkDEXE78GyTIrOB70XiDmBrSTs32+bEbjawAae8mVle6nQDLz39SO6YM2n7PT5G0kMdMy8i5rWwu8nAyprnw+m63zSqUETQ5aWnH8lddsPt/qytOq2Ub6fOWLsmTpqcu866tavaqtPOsRTRLmj9fWllH2P76XWddo8FWv9/1s6xFPF9aadOUe9LkdIA20qQ7VghQdfMrDCjI0XubRUwteb5lHRdQx7TNbNqGVmXf+ncfODo9CqG/YHnI6Lh0ALk6OlKei3JYPHY74RVwPyIWN5pa83Mui1itGvbknQ5cCCwnaRh4PPAhsl+4tvAAuAQYAXwInBc1jabBl1JnwOOBK4A7kpXTwEul3RFRJzT1pGYmfXKaPeCbkQcmfF6AJ9oZZtZPd3jgf8WES/VrpT0dWApMG7QlTREekZw7ty5HPf+g1ppk5lZ+7rY0+2FrKA7CuwCPFa3fuf0tXHVnRGMVs/Gm5m1rdgTaS3LCronA7dIeog/XYu2K/Bq4MReNszMrC2D3NONiBsk7Qnsx8tPpC2MiHL/OTGz9VJ056qEnlEBc6Q5I83M8uo4I+2PD/1n7piz0fS/7Hh/rSokOaIKmU/OYipnFtNYnbIef1HZhWX9XPYlI22QhxfMzAbOgJ9IMzMbLO7pmpkVqOQn0hx0zaxaupiR1gsOumZWKWW/mtVB18yqxWO6ZmYF8vCCmVmBSt7TdUaamZVJxxlif7jr6twxZ+P9PljNjDQzs8J4eKG86aZFpY6WdWLKKqSOtlOnSsdf9mNpp10dK/nwgnu6ZlYtJe/pZk5MKem1kmZK2rxu/azeNcvMrE2jo/mXPmgadCV9Cvg34JPAEkmza17+Ui8bZmbWjhh5KffSD1k93ROAfSPifSQzYv4vSSelrzU86ydpSNIiSYvmzZvXqJiZWffFaP6lD7LGdDeIiN8BRMSjkg4ErpG0G02CrudIM7O+GfAx3Scl7T32JA3A7wW2A97Qy4aZmbVlwHu6RwMvu09aRKwDjpY0t2etMjNrV8l7ulkTUw43ee0/ut8cM7MOlfw6XacBm1mZdJyW+/sffyN3zNnk0JOrmQbcToZNWbNlypqRVtZjqdrElGU7/k4y0so6yWbHSt7TdUaamVXLII/pmpkNHPd0zcwK5J6umVmB3NM1MyvQOk/BbmZWnN5fBtsRB10zqxaP6ZqZFajkQdcZaWZWJp1npF12ev6MtI98sen+0skavglMAC6MiHPqXt8V+C6wdVrm1IhY0Gyb7umaWbWMjHRlM5ImAOcDBwPDwEJJ8yNiWU2xfwSuiohvSdoLWABMa7ZdpwHn1G4aKJQ33bKs70uVUmer9LkcmIkpuze8sB+wIiIeAZB0BTAbqA26AWyZPt4KeCJro+7pmlm1tBB0JQ0BQzWr5qWTMABMBlbWvDYMvKluE2cCN0n6JLAZcFDWPjMnphynkd9rtY6ZWWFauIl5RMyLiBk1S6vzix0JfCcipgCHAJdKahpXm/Z0Jc2vXwW8Q9LWABFxWIsNNDPrqRjt2rn7VcDUmudT0nW1jgdmAUTELyRtTDKzzppGG80aXphCMn5xIcnYhYAZwLnNKtV22efO9QQTZlag7o3pLgSmS9qdJNgeARxVV+ZxYCbwHUmvAzYGnmq20azhhRnA3cDpwPMRcSvw+4i4LSJua1Sptss+NDTUqJiZWfeNjORfmkinJjsRuBFYTnKVwlJJZ0sa+5X/d8AJkhYDlwPHRsZ1uFnT9YwC50m6Ov33yaw6ZmZ91cXkiPSa2wV1686oebwMeEsr28wVQNO50j4o6VDghVZ2YGZWKGekOSPNzHLrOCPtxW98LHfM2fTkudWcI83MrDAl7+kWEnSrNJljEVlcVZqYstfH0k6dKmYXlnWSzf5kpJX7x7V7umZWLV2690KvOOiaWaWEhxfMzArk4QUzswJ5YkozswK5p2tmVqB1PpFmZlYcDy+YmRWo5MMLTgM2szLpOC33d6cdnjvmbP7la6uZBlylbJmyZqSVdY6wXh8LlPu9LGq+typ9LjtW8p6uhxfMrFocdM3MClTyNOCmM0dIepOkLdPHm0g6S9IPJc2RtFUxTTQzyy9GI/fSD1nT9VwMvJg+/ibJvO5z0nWX9LBdZmbtGY38Sx9kBd0N0nmCAGZExMkR8X8i4iyg4X3YJA1JWiRp0bx5rc5obGbWgdHR/EsfZI3pLpF0XERcAiyWNCMiFknaE3ipUaV07vixaButnvU0M2tbyU+kZfV0/wY4QNLDwF7ALyQ9AlyQvmZmVi4lH17Img34eeDY9GTa7mn54Yh4sojGmZm1KkbKnQbsjDQzK5OOM8ReOP7g3DFny4turmZGmplZUfp1KVhepU0DLiLdtKjJHFutU+Y02CKOpajjX9/T08uW0u00YDOzQVTuIV0HXTOrllhX7qjroGtm1VLumOuga2bV4hNpZmZFck/XzKw47umamRWp5D1dZ6SZWZl0nCH2zKEH5I452/74NmekmZl1ouQzsBcTdKuULVPW7LoqHYuzC6tzLP3JSOvOZgAkzSKZwGECcGFEnDNOmb8GziT5Vb84Io5qtk33dM2sUrrV05U0ATgfOBgYBhZKmh8Ry2rKTAdOA94SEc9J2iFru1lzpE2SdLSkg9LnR0n6Z0mfkLRhJwdkZtYLMZp/ybAfsCIiHomItcAVwOy6MicA50fEcwARsSZro1k93UvSMptKOgbYHLgOmJk26JjMZpuZFShG8p8bkzQEDNWsmpfOfAMwGVhZ89ow8Ka6TeyZbuc/SIYgzoyIG5rtMyvoviEi/lzSRGAVsEtEjEi6DFic50Dmzp2bsQszs+5pZXihbmqxdkwEpgMHAlOA2yW9ISJ+26hC5sSUkiYBWwCbkswGDLAR0HB4ISLmRcSMiJgxNDTUqJiZWdfFqHIvGVYBU2ueT0nX1RoG5kfESxHxa+BBkiDcUFbQvQi4H7gXOB24WtIFwEKS8Q0zs1Lp4pjuQmC6pN3TzucRwPy6MteT9HKRtB3JcEPTy0Ky5kg7T9KV6eMnJH0POAi4ICLuymyymVnBIrqT7xAR6ySdCNxIMl57cUQslXQ2sCgi5qevvUvSMmAE+ExEPNNsu5mXjEXEEzWPfwtc08FxmJn1VDeTIyJiAbCgbt0ZNY8D+HS65OI0YDMrk467qY/PmJk75uy66JZqpgGv7xlpVZpXq9U6RWWkFXEsUL6MvCIz0gZljrQcJ8j6yhlpZlYpDrpmZgXq/YhpZxx0zaxS3NM1MytQty4Z6xUHXTOrlJEW7r3QDw66ZlYp7umamRXIY7pmZgUq+9ULzkgzszLpuJu6bI9Dc8ecvR7+cTUz0szMijIymnXzxP4qbRpwEamzVZrMsaxpsL1OaW5nP2U//ip9LttpV6fKPrzgnq6ZVcqor14wMyuOLxkzMyvQejm84Ikpzaxfyj680PQ0n6StJJ0j6X5Jz0p6RtLydN3Wjep5Ykoz65eR0Q1yL/2QtdergOeAAyNim4jYFnhHuu6qXjfOzKxV0cLSD1lBd1pEzImI1WMrImJ1RMwBdutt08zMWjcayr30Q1bQfUzSZyXtOLZC0o6SPges7G3TzMxaF6HcSz80TQOW9CrgVGA2sEO6+kmSud/PiYjncuyj5OcSzaxEOo6E/77TB3LHnLetvqZcacBpUP1curyMpOOAS3LtpKQZaWWdzNFZTOXNSCtiYs6yTpg6MBlpncftnurk9N1ZXWuFmVmXrAvlXvqhaU9X0n2NXgJ2bPCamVnflL2nm5UcsSPwbpJLxGoJ+M+etMjMrAOj/W5Ahqyg+yNg84i4t/4FSbf2pEVmZh0Y6J5uRBzf5LWjut8cM7PODHpP18xsoIwMck/XzGzQlHxeSgddM6uW0ZL3dD0xpZmVSccR8/qdjsodc963+gflykjr2k4qlPlTpXm1qvC+jO2nSu9lUZmSZcsu7FZGmk+kmZkVaFTlHl5w0DWzShnpdwMylHuCeDOzFo0q/5JF0ixJD0haIenUJuUOlxSSZmRt0z1dM6uUbl29IGkCcD5wMDAMLJQ0PyKW1ZXbAjgJuDPPdrPmSNtS0pclXSrpqLrX/qVJvSFJiyQtmjdvXp52mJl1RRen69kPWBERj0TEWuAKknuL1/sCMAf4Q572ZQ0vXEJyCce1wBGSrpW0Ufra/o0qeWJKM+uXVoYXajuI6VIbsCbz8hlyhtN1/0XSPsDUiPhx3vZlDS/sERGHp4+vl3Q68DNJh+XdgZlZkVq5ZCwi5gFt/RyXtAHwdeDYVuplBd2NJG0QEaNpA78oaRVwO7B5Ow01M+ulke5dMbYKmFrzfEq6bswWwOuBW5VcprYTMF/SYRGxqNFGs+ZI+wpwU0T8tG79LOCfImJ6joY7I83M8uo4ZF4w5SO5Y84Jw5c13J+kicCDwEySYLsQOCoiljYofyvw980CLmTf2vGzDdbfIOlLzeqamfVDtzLSImKdpBOBG4EJwMURsVTS2cCiiJjfznbbvveCpMcjYtccRaNKKYpVmpiyCu9LO3WqmDpboWPpuKf77an5e7p/u7JxT7dXPEeamVXKoN97wXOkmdlAKXsasOdIM7NKGeibmHuONDMbNIM+vGBmNlAcdM3MClT2xAAHXTOrlIEe0zUzGzRlv3rBE1OaWZl03E/94m4fzh1zTn/s++VKjuiWdjJsisiWKWsWV1GTGRaR+VTmjLQi6pQ5u7CITElPTPlKHl4ws0op+09rB10zqxT3dM3MCrRO5e7rtjwbsKQdcpTxHGlm1hddnCOtJ7LuMrZN/SrgLklvJLny4dnx6tVNgRGtnhgxM2vXoA8vPA08VrduMnAPyR+K7pxuNDPrktGSn0rLGl74DPAAcFhE7B4RuwPD6WMHXDMrnYEeXoiIcyVdCZwnaSXwecp/RYaZrcfKPryQOyMtnXb9H4BpEbFTC/twkDazvDrOEDtl2hG5Y855j15R3oy0iJgv6WZgDwBJx0XEJbl2UtIsriplpBXRLihvRlpRx1/WTMmyHosz0l6ppUvGIuL3EbEkfXpWD9pjZtaRaOG/fvDElGZWKWXv6XpiSjOrlLJfMuaJKc2sUsodcj0xpZlVzLqSh13f8MbMKqVfJ8jyctA1s0oZ9BNpZmYDpew9Xc+RZmZl0nGG2DHTDs8dc7776LXlzUgzMxsEI73vSHaktBNTlnXSvLIeS1nTgHt9LO3UKTKlu6jU2Sqlp3dq0K/TNTMbKGUf03XQNbNK8dULZmYFKvvwQjsTU26bo4wnpjSzvujmXcYkzZL0gKQVkk4d5/VPS1om6T5Jt0jaLWubWXcZOwf4WkQ8LWkGcBUwKmlD4OiIuG3cg/bElGbWJ926ekHSBOB84GBgGFgoaX5ELKsp9ktgRkS8KOnjwFeADzXbblZP99CIeDp9/FXgQxHx6rQR57ZxHGZmPTVK5F4y7AesiIhHImItcAUwu7ZARPw8Il5Mn94BTMnaaFbQnShprDe8SUQsTHf0ILBR1sbNzIo22sJSOxSaLkM1m5oMrKx5Ppyua+R44CdZ7cs6kfYvwIJ0mOEGSd8ErgPeCbzido9mZv3WyiVjdUOhbZP0EWAGcEBm2aw0YEkHAh8H9iQJ0iuB64GLI2JdjvaU+1SimZVJx2m5h+x6SO6Ys+DxBQ33J+nNwJkR8e70+WkAEfHlunIHAf8EHBARa7L2mXnJWETcCtw6ToOOA3JNTFmFCRCrmJFWRJ1eZwpC9SZzLGt2XVFZn53q4v1kFgLTJe0OrAKOAF52H3FJbwTmArPyBFxo45KxGp6Y0sxKZ4TIvTST/pI/EbgRWA5cFRFLJZ0t6bC02FeBzYGrJd0raX5W+zwxpZlVSjeTIyJiAbCgbt0ZNY8PanWbnpjSzCqlgNvVdsQTU5pZpZQ9DdgTU5pZpfguY2ZmBfJNzM3MCjTQwwtmZoOm7EHXE1OaWZl0nJG2/y4H5o45dzxxazUnpiwi86eobJkisoWqNK9WrzOyxtpWpSyuKs1314+MtLL3dD28YGaV4qsXzMwKNBLlniXNQdfMKmXQM9LMzAZK2cd0m95lTNIMST+XdJmkqZJulvS8pIXpLc0a1fPElGbWF92cmLIX8swc8Xlga5Ib3JwSEQdLmpm+9ubxKnliSjPrl9GSDy9k3U93w4j4SURcDkREXEPy4BZg4563zsysRYPe0/2DpHcBWwEh6X0Rcb2kA4CR3jfPzKw1Zb96oWlGmqS/IJnHfRQ4hWSutGNIpq44ISLy3FO33H19MyuTjjPE9tx+Ru6Y8+BTiwrPSGs7DVjScRGRZ440B10zy6vjIDh9+31zx5yHnrp7oNKAzyLnxJTre+psWSemrELq6FjbqpQGvb5/LjtV9hNpniPNzCpl0NOAPUeamQ2UkSj3OX7PkWZmlTLQacCeI83MBk3Z04B97wUzq5SB7umamQ2agb56wcxs0Az61QtmZgNloNOAu6Tcf3bMrEw6zhDbbss9c8ecp194cKAy0vLvxBMAtrQfZzGV97305zK/Vo/fGWlmZgPIVy+YmRXI1+mamRWo7D3drDnStpJ0jqT7JT0r6RlJy9N1WxfVSDOzvEZiNPfSD1nT9VxFcrObAyNim4jYFnhHuu6qRpU8MaWZ9ctoRO6lH7KC7rSImBMRq8dWRMTqiJgD7NaoUkTMi4gZETFjaGioW201M8sUEbmXfsgKuo9J+qyk/7p3rqQdJX0OWNnbppmZta6bE1NKmiXpAUkrJJ06zusbSboyff1OSdOytpkVdD8EbAvcJuk5Sc8CtwLbAH+d2WIzs4J1q6craQJwPvAeYC/gSEl71RU7HnguIl4NnAfMyWpfZkaapNcCU4A7IuJ3NetnRcQNWTvAGWlmll/HGWITJ03OHXPWrV3VcH+S3gycGRHvTp+fBhARX64pc2Na5heSJgKrge2jSWDNunrhU8C/AScCSyTNrnn5S9mHlGym0SLpY81e77R8meuUtV0+lnK2az06lo6tW7tKeZfak/7pUnsSajIvH0YdTtcxXpmIWAc8TzI60FhG1/tXJDNHAEwDFgEnpc9/2Uo3vsH2F/WyfJnrlLVdPpZytmt9P5Z+LMAHgAtrnv8P4J/ryiwBptQ8fxjYrtl2s5IjNoh0SCEiHpV0IHCNpN3o0l8lM7OSWgVMrXk+JV03XpnhdHhhK+CZZhvNOpH2pKS9x56kAfi9wHbAG/K128xsIC0EpkvaXdIk4Ahgfl2Z+cAx6eMPAD+LtMvbSFZP92hgXe2KSMYtjpY0N2/Lm2g1c6KdTIuy1ilru9qpU9Z2tVOnrO1qp05Z29VunUJFxDpJJwI3AhOAiyNiqaSzSYZH5gMXAZdKWgE8SxKYmyrifrpmZpbKGl4wM7MuctA1MytQX4JuVmrdOOUvlrRG0pIW9jFV0s8lLZO0VNJJGeU3lnSXpMVp+bNa2NcESb+U9KOc5R+V9CtJ90palLPO1pKuSe/4tjy9cLtR2dek2x5bXpB0co59nJIe+xJJl0vaOEedk9LySxvtY7z3T9I2km6W9FD676ty1Plgup9RSTNylP9q+v/rPkn/qro74zWo84W0/L2SbpK0S1admtf+TlJI2i7Hfs6UtKrmPTokax+SPpkez1JJX8mxjytrtv+opHtz1Nlb0h1jn01J++Wo8xeSfpF+pn8oacua18b9Hma9/5XWh2vfJpBcy/ZnwCRgMbBXRp23A/sAS1rYz87APunjLYAHm+2H5BK4sWuSNwTuBPbPua9PAz8AfpSz/KNkXMs3Tp3vAn+TPp4EbN3C/+/VwG4Z5SYDvwY2SZ9fBRybUef1JNcpbkpyUvanwKvzvH/AV4BT08enAnNy1Hkd8BqSVPQZOcq/C5iYPp6Tcx9b1jz+FPDtPJ9FksuGbgQeq39vG+znTODv837eSe7u91Ngo/T5Dq18R4BzgTNy7Ocm4D3p40OAW3PUWQgckD7+KPCFmtfG/R5mvf9VXvrR090PWBERj0TEWuAKYHazChFxO8mZwdwi4jcRcU/6+P8Cy3llNklt+Yg/pTlvmC6ZZxklTQEOBS5spX2tkLQVyYf9IoCIWBsRv81ZfSbwcEQ8lqPsRGATJdcbbgo8kVH+dcCdEfFiJFe13Aa8v75Qg/dvNskfEtJ/35dVJyKWR8QD4zWkQfmb0nYB3EFynWVWnRdqnm5G3WegyWfxPOCz9eUz6oyrQfmPA+dExB/TMmvy7kOSSO6VcnmOOgGM9VS3ou4z0KDOnsDt6eObgcNryjf6HjZ9/6usH0E3T2pdV0maBryRpPfarNyE9CfYGuDmiGhaPvUNki9bK3dEDuAmSXfr5WmHjewOPAVckg5jXChps5z7OoK6L9u4DYpYBXwNeBz4DfB8RNyUUW0J8DZJ20ralKRnNDWjzpgdI+I36ePVwI7NCnfBR4Gf5Cko6YuSVgIfBs7IUX42sCoiFrfYphPToYyLc/y83pPk//Wdkm6T9N9b2M/bgCcj4qEcZU8Gvpoe/9eA03LUWcqfOk4fpMFnoO57WPT7XxqVP5EmaXPgWuDkul7MK0TESETsTdIj2k/S6zO2/V5gTUTc3WKz3hoR+5DcvegTkt6eUX4iyU+6b0XEG4H/R/KTrCklF3QfBlydo+yrSL44uwO7AJtJ+kizOhGxnORn+03ADcC9wEjWvsbZTtDDGyNJOp3kevPv52zP6RExNS1/Ysa2NwX+gRzBuc63gD2AvUn+yJ2bUX4iyd399gc+A1yV9mDzOJIcf3hTHwdOSY//FNJfVxk+CvxPSXeTDCGsrS/Q7HvY6/e/bPoRdPOk1nWFpA1J3ujvR8R1eeulP91/DszKKPoW4DBJj5IMk7xT0mU5tr8q/XcN8K8kQy7NDAPDNT3va0iCcJb3APdExJM5yh4E/DoinoqIl4DrgL/MqhQRF0XEvhHxdpIZRR7MsS9Ish13Bkj/XZNRvi2SjiXJovxw+uVuxfep+ancwB4kf6gWp5+DKcA9knZqVikinkz/yI8CF5DvM3BdOgx2F8kvq+0y6pAOFb0fuDKrbOoYkvcekj/WWe0iIu6PiHdFxL4kwf3hujaM9z0s5P0vo34E3TypdR1LewEXAcsj4us5ym8/dnZb0ibAwcD9zepExGkRMSUippEcx88iomnvUNJmkrYYe0xysqfpVRmRzNyxUtJr0lUzgWVZx0RrPZzHgf0lbZr+v5tJMv7WlKQd0n93Jfly/yDn/mrTJ48huZtdV0maRTL0c1hEvJizzvSap7PJ/gz8KiJ2iIhp6edgmOTE0epm9cYCTuqvyPgMANeTnExD0p4kJ1OfzqgDyR/T+yNiOEdZSMZwD0gfvxPIHJKo+QxsAPwj8O2a1xp9D3v+/pdWP87ekYz9PUjyF/H0HOUvJ/kJ9hLJh/r4HHXeSvKT5T6Sn733Aoc0Kf/nwC/T8kuoO9ObY38HkuPqBZKrNhany9I8x5/W25vkLm/3kXwBX5VRfjOSG29s1cIxnEUSZJYAl5KeKc+o8+8kfwAWAzPzvn8kt7+7heRL/VNgmxx1/ip9/EfgSeDGjPIrSM4fjL3/9VcijFfn2vT47wN+CExu5bPIOFemNNjPpSR38buPJADtnFF+EnBZ2rZ7gHfmaRfwHeBvW3hf3grcnb6fdwL75qhzEsn3+UHgHNJM12bfw6z3v8qL04DNzApU+RNpZmZl4qBrZlYgB10zswI56JqZFchB18ysQA66ZmYFctA1MyvQ/wcFn9UyDT3+XwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ax = sns.heatmap(action_map, linewidth=0.5)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9977138408260168\n"
     ]
    }
   ],
   "source": [
    "print(np.max(J))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(100):\n",
    "    s = env.reset()\n",
    "    done = False\n",
    "    while not done:\n",
    "        a = random.choice([1,-1])\n",
    "        ns, c, done,_ = env.step(a)\n",
    "        s = ns\n",
    "        env.render()\n",
    "        time.sleep(0.05)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
